{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python383jvsc74a57bd00b98fe7045fac7b56c2439711bd67da1839d9b049b6e8862420bcf34ecf3796f",
   "display_name": "Python 3.8.3 64-bit ('base': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = r'https://etherscan.io/txs?a=0x6BEcAb24Ed88Ec13D0A18f20e7dC5E4d5b146542&p=1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {\n",
    "'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get(url, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('res.txt', 'w') as f:\n",
    "    f.write(res.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "tree = etree.HTML(res.text) # 或者直接用html数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn_hash = tree.xpath('//*[@class=\"myFnExpandBox_searchVal\"]/text()') # 获取文本内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "method = tree.xpath('//*[@id=\"paywall_mask\"]/table/tbody/tr[3]/td[3]/span/@title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "method = []\n",
    "for i in range(1, len(txn_hash) + 1):\n",
    "    xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[3]/span/@title'\n",
    "    method.extend(tree.xpath(xpath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "//*[@id=\"paywall_mask\"]/table/tbody/tr[1]/td[4]/a\n",
    "//*[@id=\"paywall_mask\"]/table/tbody/tr[2]/td[4]/a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "block = []\n",
    "for i in range(1, len(txn_hash) + 1):\n",
    "    xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[4]/a/text()'\n",
    "    block.extend(tree.xpath(xpath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "age = []\n",
    "for i in range(1, len(txn_hash) + 1):\n",
    "    xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[6]/span/text()'\n",
    "    age.extend(tree.xpath(xpath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "//*[@id=\"paywall_mask\"]/table/tbody/tr[1]/td[7]/span\n",
    "//*[@id=\"paywall_mask\"]/table/tbody/tr[3]/td[7]/span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "from_ = []\n",
    "for i in range(1, len(txn_hash) + 1):\n",
    "    xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[7]/span/text()'\n",
    "    from_.extend(tree.xpath(xpath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "//*[@id=\"paywall_mask\"]/table/tbody/tr[1]/td[9]/span/a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_ = []\n",
    "for i in range(1, len(txn_hash) + 1):\n",
    "    xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[9]/span/a/text()'\n",
    "    if tree.xpath(xpath):\n",
    "        to_.extend(tree.xpath(xpath))\n",
    "    else:\n",
    "        # 这里会出现问题，所以用else\n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[9]/span/span/a/text()'\n",
    "        to_.extend(tree.xpath(xpath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "value = []\n",
    "for i in range(1, len(txn_hash) + 1):\n",
    "    xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[10]//text()'\n",
    "    temp = ''.join(tree.xpath(xpath)) # 这里使用字符串进行拼接\n",
    "    value.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "txn_free = []\n",
    "for i in range(1, len(txn_hash) + 1):\n",
    "    xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[11]/span/text()'\n",
    "    temp = '.'.join(tree.xpath(xpath))\n",
    "    txn_free.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "正在爬取第1页...\n",
      "正在爬取第2页...\n",
      "正在爬取第3页...\n",
      "正在爬取第4页...\n",
      "正在爬取第5页...\n",
      "正在爬取第6页...\n",
      "正在爬取第7页...\n",
      "正在爬取第8页...\n",
      "正在爬取第9页...\n",
      "正在爬取第10页...\n",
      "正在爬取第11页...\n",
      "正在爬取第12页...\n",
      "正在爬取第13页...\n",
      "正在爬取第14页...\n",
      "正在爬取第15页...\n",
      "正在爬取第16页...\n",
      "正在爬取第17页...\n",
      "正在爬取第18页...\n",
      "正在爬取第19页...\n",
      "正在爬取第20页...\n",
      "正在爬取第21页...\n",
      "正在爬取第22页...\n",
      "正在爬取第23页...\n",
      "正在爬取第24页...\n",
      "正在爬取第25页...\n",
      "正在爬取第26页...\n",
      "正在爬取第27页...\n",
      "正在爬取第28页...\n",
      "正在爬取第29页...\n",
      "正在爬取第30页...\n",
      "正在爬取第31页...\n",
      "正在爬取第32页...\n",
      "正在爬取第33页...\n",
      "正在爬取第34页...\n",
      "正在爬取第35页...\n",
      "正在爬取第36页...\n",
      "正在爬取第37页...\n",
      "正在爬取第38页...\n",
      "正在爬取第39页...\n",
      "正在爬取第40页...\n"
     ]
    }
   ],
   "source": [
    "pages = 40 # 一共si shi\n",
    "method = []\n",
    "block = []\n",
    "age = []\n",
    "from_ = []\n",
    "to_ = []\n",
    "value = []\n",
    "txn_free = []\n",
    "\n",
    "for page in range(1, pages+1):\n",
    "    print(f'正在爬取第{page}页...')\n",
    "    url = f'https://etherscan.io/txs?a=0x6BEcAb24Ed88Ec13D0A18f20e7dC5E4d5b146542&p={page}'\n",
    "    res = requests.get(url, headers=headers)\n",
    "    tree = etree.HTML(res.text) # 或者直接用html数据\n",
    "    txn_hash = tree.xpath('//*[@class=\"myFnExpandBox_searchVal\"]/text()') # 获取文本内容\n",
    "    for i in range(1, len(txn_hash) + 1):\n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[3]/span/@title'\n",
    "        method.extend(tree.xpath(xpath))\n",
    "    \n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[4]/a/text()'\n",
    "        block.extend(tree.xpath(xpath))\n",
    "    \n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[6]/span/text()'\n",
    "        age.extend(tree.xpath(xpath))\n",
    "    \n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[7]/span/text()'\n",
    "        from_.extend(tree.xpath(xpath))\n",
    "    \n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[9]/span/a/text()'\n",
    "        if tree.xpath(xpath):\n",
    "            to_.extend(tree.xpath(xpath))\n",
    "        else:\n",
    "            # 这里会出现问题，所以用else\n",
    "            xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[9]/span/span/a/text()'\n",
    "            to_.extend(tree.xpath(xpath))\n",
    "    \n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[10]//text()'\n",
    "        temp = ''.join(tree.xpath(xpath)) # 这里使用字符串进行拼接\n",
    "        value.append(temp)\n",
    "    \n",
    "   \n",
    "        xpath = f'//*[@id=\"paywall_mask\"]/table/tbody/tr[{i}]/td[11]/span/text()'\n",
    "        temp = '.'.join(tree.xpath(xpath))\n",
    "        txn_free.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(1999, 1999, 1999, 1995, 1987, 1999, 1999)"
      ]
     },
     "metadata": {},
     "execution_count": 143
    }
   ],
   "source": [
    "len(method), len(block), len(age), len(from_), len(to_), len(value), len(txn_free)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 可以看出from to有空的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    'method':method,\n",
    "    'block':block,\n",
    "    'age':age,\n",
    "    'from_':from_,\n",
    "    'to_':to_,\n",
    "    'value':value,\n",
    "    'txn_free':txn_free,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "ValueError",
     "evalue": "arrays must all be same length",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-150-cd168f76d566>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m    527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 529\u001b[0;31m             \u001b[0mmgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minit_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    530\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMaskedArray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmrecords\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmrecords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36minit_dict\u001b[0;34m(data, index, columns, dtype)\u001b[0m\n\u001b[1;32m    285\u001b[0m             \u001b[0marr\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_datetime64tz_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0marr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         ]\n\u001b[0;32m--> 287\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marrays_to_mgr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36marrays_to_mgr\u001b[0;34m(arrays, arr_names, index, columns, dtype, verify_integrity)\u001b[0m\n\u001b[1;32m     78\u001b[0m         \u001b[0;31m# figure out the index, if necessary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m             \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m             \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36mextract_index\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m    399\u001b[0m             \u001b[0mlengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_lengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    400\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlengths\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 401\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"arrays must all be same length\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    402\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    403\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhave_dicts\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: arrays must all be same length"
     ]
    }
   ],
   "source": [
    "pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}